#!/usr/bin/perl
#
# Count referers and their source IP
# Flag the ones that look suspicious.
#
use Getopt::Long;

$printlimit = 100; # this many or more gets printed - suspicious
$quiet = 0;

GetOptions("limit=i" => \$printlimit,
	   "quiet" => \$quiet,
	   );


# About half the log is inernal referers, so throw it away at once
open(LOG, "grep -v 'Referer:http://counter.li.org' /var/log/httpd/referer_log|");
while (<LOG>) {
    if (m!GET /cgi-bin/certificate.cgi!) {
	++ $certreq;
	next;
    }
    # References that come from other affiliated sites are OK
    if (m!Referer:http://(www.linuxcounter.org|lugww.counter.li.org)/!) {
	next;
    }
    # one robot that puts his name into Referer
    if (m!Referer:http://www.miragorobot.com/!) {
	next;
    }
    # check for self-referers generically (useful for offbeat sitenames like i18n2)
    if (m!Referer:http://([^/]+)/!) {
	my $fromsite = $1;
	if (m!\] $fromsite \"!) {
	    ++ $selfreferers;
	    next;
	}
    }
    if (/^(\S+) .*Referer:(\S+)$/) {
	if ($2 eq "-") {
	    # Another half is "no referer". Junk them.
	    ++ $noreferer;
	    next;
	}
	++ $references{$1,$2};
	++ $iprefs{$1};
    }
}
!$quiet && warn "$certreq certificate requests\n";
!$quiet && warn "$noreferer no referer\n";
!$quiet && warn "$selfreferers self-referers\n";
for $ref (sort(keys(%references))) {
    ($ip, $referer) = split(/$;/, $ref);
    if ($references{$ref} < $printlimit) {
	++ $belowlimit;
    } else {
	print $references{$ref}, " ", $ip, " ", $referer, "\n";
    }
}
!$quiet && warn "$belowlimit entries with < $printlimit occurences\n";
for $ref (sort(keys(%iprefs))) {
    if ($iprefs{$ref} < $printlimit) {
	++ $ipbelowlimit;
    } else {
	print $iprefs{$ref}, " ", $ref, "\n";
    }
}
!$quiet && warn "$ipbelowlimit IP entries with < $printlimit occurences\n";

